package org.newlucene.core.analysis;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class SimpleAnalyzer implements Analyzer
{
   private String regexp = "[a-z0-9A-Z]+";
   private Pattern pattern = Pattern.compile(regexp);
   private Matcher matcher;

   // always converts tokens to lower case before returning
   public SimpleAnalyzer()
   {
       this.matcher = pattern.matcher("");
   }
   
   public void set(String text)
   {
      this.matcher.reset(text);
   }

   public Token next()
   {
      if (matcher.find())
      {
         String text = matcher.group();
         return new Token(text.toLowerCase());
      }
      else
      {
         return null;
      }
   }   
}